import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;

public class CalculateVectorSize {

	/**
	 * @param args
	 */
	public static void main(String[] args) {
		//Read dictionary
			ArrayList<String> dictionary = new ArrayList<String>();
			String fileDicName = "dictionary.txt";
			try {
				File fileDir = new File(fileDicName);

				BufferedReader in = new BufferedReader(new InputStreamReader(
						new FileInputStream(fileDir), "UTF8"));

				String str;

				while ((str = in.readLine()) != null) {
					dictionary.add(str);
				}

				in.close();
			} catch (UnsupportedEncodingException e) {
				System.out.println(e.getMessage());
			} catch (IOException e) {
				System.out.println(e.getMessage());
			} catch (Exception e) {
				System.out.println(e.getMessage());
			}
			java.util.Collections.sort(dictionary);

			//---------------------------------------------
				
		ArrayList<Article> docs = new ArrayList<Article>();
		
		String vectorFile = "data_doisong_temp.txt";
		String fileWithoutPuctuation= "";
		int numOfArticle = 1100;
		String regex = "[0-9]+"; 
		
		for(int i = 0; i < numOfArticle; i++){
			String fileName = "";
			fileName = "doisong (" + (i+1) + ").txt";
//			if((i + 1) < 10){
//				fileName = "vnexpress000" + (i+1) + ".doisong";
//			} else if((i + 1)/10 >= 1 && (i + 1)/10 < 10){
//				fileName = "vnexpress00" + (i+1) + ".doisong";
//			} else if((i + 1)/100 >= 1 && (i + 1)/100 < 10){
//				fileName = "vnexpress0" + (i+1) + ".doisong";
//			} else if((i + 1)/1000 >= 1 && (i + 1)/1000 < 10){
//				fileName = "vnexpress" + (i+1) + ".doisong";
//			}
			System.out.println(fileName);
			
			fileWithoutPuctuation = readFile(fileName);
			//remove special characters
			fileWithoutPuctuation = fileWithoutPuctuation.replaceAll("[^\\p{L}\\p{Nd}]"," ");
			
			String[] articleWords = fileWithoutPuctuation.split(" ");
			HashSet<String> hashSetAW = new HashSet<String>();
			for(String word : articleWords){
				hashSetAW.add(word);
			}
			
			for(String word : hashSetAW){
				int checkLength = -1;
				for(int k = 0; k < docs.size(); k++){
					if(docs.get(k).getWord().equals(word.trim())){
						docs.get(k).setNumOfWord(docs.get(k).getNumOfWord() + 1);
						checkLength = k;
						break;
					} 
				}
				//check if word is in Vietnamese dictionary
				if(checkLength == -1 && dictionary.contains(word)){
					Article article = new Article();
					article.setWord(word.trim());
					article.setNumOfWord(article.getNumOfWord() + 1);
					docs.add(article);
				}
			}
		}		
		
//		docs = sortArrayList(docs);
		
		List<String> articles = new LinkedList<String>();
		
		String writeString = "";
		for(int i = 0; i < docs.size(); i++){
//			writeString += docs.get(i).getWord() + " " + docs.get(i).getNumOfWord() + "\r\n";
			articles.add(docs.get(i).getWord() + " " + docs.get(i).getNumOfWord() + "\r\n");
		}
		
		java.util.Collections.sort(articles);
		
		for (String name : articles) {
			writeString += name;
	        System.out.println(name);
	    }
		
		writeFile(vectorFile, writeString);
			

	}

	static String readFile(String fileName) {
		String temp = "";
		try (BufferedReader br = new BufferedReader(new FileReader(fileName))) {
			String sCurrentLine;
			while ((sCurrentLine = br.readLine()) != null) {
				temp += sCurrentLine;
			}
		} catch (IOException e) {
			e.printStackTrace();
		}

		return temp;
	}
	
	static void writeFile(String fileName, String writeString) {
		try {
			File fileDir = new File(fileName);
			BufferedWriter out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(fileDir), "UTF8"));
			
			out.append(writeString);
			
			out.flush();
			out.close();
		} catch (UnsupportedEncodingException e) {
			System.out.println(e.getMessage());
		} catch (IOException e) {
			System.out.println(e.getMessage());
		} catch (Exception e) {
			System.out.println(e.getMessage());
		}
	}
	
	static String filterLongWord(String article){
		String temp = "";
		String[] tempArray;
		tempArray = article.split(" ");
		
		for(int i = 0; i < tempArray.length; i++){
			if(tempArray[i].length() < 8){
				temp += tempArray[i] + " ";
			}			
		}
		
		return temp;
	}
	
	static ArrayList<Article> sortArrayList(ArrayList<Article> arrayList){
		for(int i = 0; i < arrayList.size() - 1; i++){
			Locale locales[] = Locale.getAvailableLocales();
			System.out.println(locales);
			
			int min = i;
			for(int j = i + 1; j < arrayList.size(); j++){
				if(arrayList.get(i).getWord().compareTo(arrayList.get(j).getWord()) > 0){
					min = j;					
				}
			}
			if(min != i){
				Article tempArticle = new Article();
				tempArticle.setWord(arrayList.get(i).getWord());
				tempArticle.setNumOfWord(arrayList.get(i).getNumOfWord());
				
				arrayList.get(i).setWord(arrayList.get(min).getWord());
				arrayList.get(i).setNumOfWord(arrayList.get(min).getNumOfWord());
				
				arrayList.get(min).setWord(tempArticle.getWord());
				arrayList.get(min).setNumOfWord(tempArticle.getNumOfWord());
			}
		}
		
		return arrayList;
	}
}
